Node / ReticulumProjects / Sideband.git / files / sbapp / mapview / downloader.py

Displaying Raw • Download

T8b949e# coding=utf-8

Te6edf3__all__ Tff7b72= Tb4b4b4[Ta5d6ff"Ta5d6ffDownloaderTa5d6ff"Tb4b4b4]

Tff7b72import T7ee787logging

Tff7b72import T7ee787traceback

Tff7b72from T7ee787concurrentT7ee787.T7ee787futures Tff7b72import Te6edf3ThreadPoolExecutorTb4b4b4, Tf85149TimeoutErrorTb4b4b4, Te6edf3as_completed

Tff7b72from T7ee787os Tff7b72import Te6edf3environTb4b4b4, Te6edf3makedirs

Tff7b72from T7ee787osT7ee787.T7ee787path Tff7b72import Te6edf3existsTb4b4b4, Te6edf3join

Tff7b72from T7ee787random Tff7b72import Te6edf3choice

Tff7b72from T7ee787time Tff7b72import Te6edf3time

Tff7b72import T7ee787requests

Tff7b72from T7ee787kivyT7ee787.T7ee787clock Tff7b72import Te6edf3Clock

Tff7b72from T7ee787kivyT7ee787.T7ee787logger Tff7b72import Te6edf3LOG_LEVELSTb4b4b4, Te6edf3Logger

Tff7b72from T7ee787mapviewT7ee787.T7ee787constants Tff7b72import Te6edf3CACHE_DIR

Tff7b72import T7ee787logging

T8b949e# if "MAPVIEW_DEBUG_DOWNLOADER" in environ:

T8b949e# Logger.setLevel(LOG_LEVELS['debug'])

T8b949e# Logger.setLevel(LOG_LEVELS['error'])

T8b949e# user agent is needed because since may 2019 OSM gives me a 429 or 403 server error

T8b949e# I tried it with a simpler one (just Mozilla/5.0) this also gets rejected

Te6edf3USER_AGENT Tff7b72= Ta5d6ff'Ta5d6ffKivy-garden.mapviewTa5d6ff'

Tff7b72import T7ee787RNS

Tff7b72class T56d364DownloaderTb4b4b4:

Te6edf3_instance Tff7b72= Tff7b72None

Te6edf3MAX_WORKERS Tff7b72= T79c0ff5

Te6edf3CAP_TIME Tff7b72= T79c0ff0.064 T8b949e# 15 FPS

Tf0883e@staticmethod

Tff7b72def Td2a8ffinstanceTb4b4b4(Te6edf3cache_dirTff7b72=Tff7b72NoneTb4b4b4)Tb4b4b4:

Tff7b72if Te6edf3DownloaderTff7b72.Td2a8ff_instance Tff7b72is Tff7b72NoneTb4b4b4:

Tff7b72if Tff7b72not Te6edf3cache_dirTb4b4b4:

Te6edf3cache_dir Tff7b72= Te6edf3CACHE_DIR

Te6edf3DownloaderTff7b72.Td2a8ff_instance Tff7b72= Te6edf3DownloaderTb4b4b4(Te6edf3cache_dirTff7b72=Te6edf3cache_dirTb4b4b4)

Tff7b72return Te6edf3DownloaderTff7b72.Td2a8ff_instance

Tff7b72def Tff7b72__init__Tb4b4b4(Tff7b72selfTb4b4b4, Te6edf3max_workersTff7b72=Tff7b72NoneTb4b4b4, Te6edf3cap_timeTff7b72=Tff7b72NoneTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8ffcache_dir Tff7b72= Te6edf3kwargsTff7b72.Td2a8ffgetTb4b4b4(Ta5d6ff'Ta5d6ffcache_dirTa5d6ff'Tb4b4b4, Te6edf3CACHE_DIRTb4b4b4)

Tff7b72if Te6edf3max_workers Tff7b72is Tff7b72NoneTb4b4b4:

Te6edf3max_workers Tff7b72= Te6edf3DownloaderTff7b72.Td2a8ffMAX_WORKERS

Tff7b72if Te6edf3cap_time Tff7b72is Tff7b72NoneTb4b4b4:

Te6edf3cap_time Tff7b72= Te6edf3DownloaderTff7b72.Td2a8ffCAP_TIME

Tff7b72selfTff7b72.Td2a8ffis_paused Tff7b72= Tff7b72False

Tff7b72selfTff7b72.Td2a8ffcap_time Tff7b72= Te6edf3cap_time

Tff7b72selfTff7b72.Td2a8ffexecutor Tff7b72= Te6edf3ThreadPoolExecutorTb4b4b4(Te6edf3max_workersTff7b72=Te6edf3max_workersTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_futures Tff7b72= Tb4b4b4[Tb4b4b4]

Te6edf3ClockTff7b72.Td2a8ffschedule_intervalTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_check_executorTb4b4b4, T79c0ff1 Tff7b72/ T79c0ff60.0Tb4b4b4)

Tff7b72if Tff7b72not Te6edf3existsTb4b4b4(Tff7b72selfTff7b72.Td2a8ffcache_dirTb4b4b4)Tb4b4b4:

Te6edf3RNSTff7b72.Td2a8fflogTb4b4b4(Ta5d6ff"Ta5d6ffCreating cache dir Ta5d6ff"Tff7b72+Tffa657strTb4b4b4(Tff7b72selfTff7b72.Td2a8ffcache_dirTb4b4b4)Tb4b4b4, Te6edf3RNSTff7b72.Td2a8ffLOG_WARNINGTb4b4b4)

Te6edf3makedirsTb4b4b4(Tff7b72selfTff7b72.Td2a8ffcache_dirTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3Ta5d6ff"Tb4b4b4)Tff7b72.Td2a8ffsetLevelTb4b4b4(Te6edf3loggingTff7b72.Td2a8ffWARNINGTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.responseTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffsetLevelTb4b4b4(Te6edf3loggingTff7b72.Td2a8ffWARNINGTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.connectionTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffsetLevelTb4b4b4(Te6edf3loggingTff7b72.Td2a8ffWARNINGTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.connectionpoolTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffsetLevelTb4b4b4(Te6edf3loggingTff7b72.Td2a8ffWARNINGTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffrequestsTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffsetLevelTb4b4b4(Te6edf3loggingTff7b72.Td2a8ffWARNINGTb4b4b4)

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3Ta5d6ff"Tb4b4b4)Tff7b72.Td2a8ffpropagate Tff7b72= Tff7b72True

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffrequestsTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffpropagate Tff7b72= Tff7b72True

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.responseTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffpropagate Tff7b72= Tff7b72True

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.connectionTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffpropagate Tff7b72= Tff7b72True

Te6edf3loggingTff7b72.Td2a8ffgetLoggerTb4b4b4(Ta5d6ff"Ta5d6ffurllib3.connectionpoolTa5d6ff"Tb4b4b4)Tff7b72.Td2a8ffpropagate Tff7b72= Tff7b72True

Tff7b72def Td2a8ffsubmitTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3fTb4b4b4, Tff7b72*Te6edf3argsTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)Tb4b4b4:

Te6edf3future Tff7b72= Tff7b72selfTff7b72.Td2a8ffexecutorTff7b72.Td2a8ffsubmitTb4b4b4(Te6edf3fTb4b4b4, Tff7b72*Te6edf3argsTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_futuresTff7b72.Td2a8ffappendTb4b4b4(Te6edf3futureTb4b4b4)

Tff7b72def Td2a8ffdownload_tileTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3tileTb4b4b4)Tb4b4b4:

T8b949e# Logger.debug(

T8b949e# "Downloader: queue(tile) zoom={} x={} y={}".format(

T8b949e# tile.zoom, tile.tile_x, tile.tile_y

T8b949e# )

T8b949e# )

Te6edf3future Tff7b72= Tff7b72selfTff7b72.Td2a8ffexecutorTff7b72.Td2a8ffsubmitTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_load_tileTb4b4b4, Te6edf3tileTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_futuresTff7b72.Td2a8ffappendTb4b4b4(Te6edf3futureTb4b4b4)

Tff7b72def Td2a8ffdownloadTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3urlTb4b4b4, Te6edf3callbackTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)Tb4b4b4:

T8b949e# Logger.debug("Downloader: queue(url) {}".format(url))

Te6edf3future Tff7b72= Tff7b72selfTff7b72.Td2a8ffexecutorTff7b72.Td2a8ffsubmitTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_download_urlTb4b4b4, Te6edf3urlTb4b4b4, Te6edf3callbackTb4b4b4, Te6edf3kwargsTb4b4b4)

Tff7b72selfTff7b72.Td2a8ff_futuresTff7b72.Td2a8ffappendTb4b4b4(Te6edf3futureTb4b4b4)

Tff7b72def Td2a8ff_download_urlTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3urlTb4b4b4, Te6edf3callbackTb4b4b4, Te6edf3kwargsTb4b4b4)Tb4b4b4:

T8b949e# Logger.debug("Downloader: download(url) {}".format(url))

Te6edf3response Tff7b72= Te6edf3requestsTff7b72.Td2a8ffgetTb4b4b4(Te6edf3urlTb4b4b4, Tff7b72*Tff7b72*Te6edf3kwargsTb4b4b4)

Te6edf3responseTff7b72.Td2a8ffraise_for_statusTb4b4b4(Tb4b4b4)

Tff7b72return Te6edf3callbackTb4b4b4, Tb4b4b4(Te6edf3urlTb4b4b4, Te6edf3responseTb4b4b4)

Tff7b72def Td2a8ff__to_quadTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3xTb4b4b4, Te6edf3yTb4b4b4, Te6edf3zTb4b4b4)Tb4b4b4:

Te6edf3quad_key Tff7b72= Tb4b4b4[Tb4b4b4]

Te6edf3i Tff7b72= Te6edf3z

Tff7b72while Te6edf3i Tff7b72> T79c0ff0Tb4b4b4:

Te6edf3digit Tff7b72= T79c0ff0

Te6edf3mask Tff7b72= T79c0ff1 Tff7b72<< Tb4b4b4(Te6edf3iTff7b72-T79c0ff1Tb4b4b4)

Tff7b72if Tb4b4b4(Te6edf3x Tff7b72& Te6edf3maskTb4b4b4) Tff7b72!= T79c0ff0Tb4b4b4:

Te6edf3digit Tff7b72+Tff7b72= T79c0ff1

Tff7b72if Tb4b4b4(Te6edf3y Tff7b72& Te6edf3maskTb4b4b4) Tff7b72!= T79c0ff0Tb4b4b4:

Te6edf3digit Tff7b72+Tff7b72= T79c0ff2

Te6edf3quad_keyTff7b72.Td2a8ffappendTb4b4b4(Tffa657strTb4b4b4(Te6edf3digitTb4b4b4)Tb4b4b4)

Te6edf3i Tff7b72-Tff7b72= T79c0ff1

Tff7b72return Ta5d6ff"Ta5d6ff"Tff7b72.Td2a8ffjoinTb4b4b4(Te6edf3quad_keyTb4b4b4)

Tff7b72def Td2a8ff_load_tileTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3tileTb4b4b4)Tb4b4b4:

Tff7b72if Te6edf3tileTff7b72.Td2a8ffstate Tff7b72== Ta5d6ff"Ta5d6ffdoneTa5d6ff"Tb4b4b4:

Tff7b72return

Te6edf3cache_fn Tff7b72= Te6edf3tileTff7b72.Td2a8ffcache_fn

Tff7b72if Te6edf3existsTb4b4b4(Te6edf3cache_fnTb4b4b4)Tb4b4b4:

T8b949e# Logger.debug("Downloader: use cache {}".format(cache_fn))

Tff7b72return Te6edf3tileTff7b72.Td2a8ffset_sourceTb4b4b4, Tb4b4b4(Te6edf3cache_fnTb4b4b4,Tb4b4b4)

Te6edf3tile_y Tff7b72= Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffget_row_countTb4b4b4(Te6edf3tileTff7b72.Td2a8ffzoomTb4b4b4) Tff7b72- Te6edf3tileTff7b72.Td2a8fftile_y Tff7b72- T79c0ff1

Tff7b72if Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffquad_keyTb4b4b4:

Te6edf3uri Tff7b72= Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffurlTff7b72.Td2a8ffformatTb4b4b4(

Te6edf3qTff7b72=Tff7b72selfTff7b72.Td2a8ff__to_quadTb4b4b4(Te6edf3tileTff7b72.Td2a8fftile_xTb4b4b4,Te6edf3tile_yTb4b4b4,Te6edf3tileTff7b72.Td2a8ffzoomTb4b4b4)Tb4b4b4, Te6edf3sTff7b72=Te6edf3choiceTb4b4b4(Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffsubdomainsTb4b4b4)

Tb4b4b4)

Tff7b72elseTb4b4b4:

Te6edf3uri Tff7b72= Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffurlTff7b72.Td2a8ffformatTb4b4b4(

Te6edf3zTff7b72=Te6edf3tileTff7b72.Td2a8ffzoomTb4b4b4, Te6edf3xTff7b72=Te6edf3tileTff7b72.Td2a8fftile_xTb4b4b4, Te6edf3yTff7b72=Te6edf3tile_yTb4b4b4, Te6edf3sTff7b72=Te6edf3choiceTb4b4b4(Te6edf3tileTff7b72.Td2a8ffmap_sourceTff7b72.Td2a8ffsubdomainsTb4b4b4)

Tb4b4b4)

T8b949e# Logger.debug("Downloader: download(tile) {}".format(uri))

Te6edf3response Tff7b72= Te6edf3requestsTff7b72.Td2a8ffgetTb4b4b4(Te6edf3uriTb4b4b4, Te6edf3headersTff7b72=Tb4b4b4{Ta5d6ff'Ta5d6ffUser-agentTa5d6ff'Tb4b4b4: Te6edf3USER_AGENTTb4b4b4}Tb4b4b4, Te6edf3timeoutTff7b72=T79c0ff5Tb4b4b4)

Tff7b72tryTb4b4b4:

Te6edf3responseTff7b72.Td2a8ffraise_for_statusTb4b4b4(Tb4b4b4)

Te6edf3data Tff7b72= Te6edf3responseTff7b72.Td2a8ffcontent

Tff7b72with Tffa657openTb4b4b4(Te6edf3cache_fnTb4b4b4, Ta5d6ff"Ta5d6ffwbTa5d6ff"Tb4b4b4) Tff7b72as Te6edf3fdTb4b4b4:

Te6edf3fdTff7b72.Td2a8ffwriteTb4b4b4(Te6edf3dataTb4b4b4)

T8b949e# Logger.debug("Downloaded {} bytes: {}".format(len(data), uri))

Tff7b72return Te6edf3tileTff7b72.Td2a8ffset_sourceTb4b4b4, Tb4b4b4(Te6edf3cache_fnTb4b4b4,Tb4b4b4)

Tff7b72except Tf85149Exception Tff7b72as Te6edf3eTb4b4b4:

Tffa657printTb4b4b4(Ta5d6ff"Ta5d6ffDownloader error: Tffd700{!r}Ta5d6ff"Tff7b72.Td2a8ffformatTb4b4b4(Te6edf3eTb4b4b4)Tb4b4b4)

Tff7b72def Td2a8ff_check_executorTb4b4b4(Tff7b72selfTb4b4b4, Te6edf3dtTb4b4b4)Tb4b4b4:

Te6edf3start Tff7b72= Te6edf3timeTb4b4b4(Tb4b4b4)

Tff7b72tryTb4b4b4:

Tff7b72for Te6edf3future Tff7b72in Te6edf3as_completedTb4b4b4(Tff7b72selfTff7b72.Td2a8ff_futuresTb4b4b4[Tb4b4b4:Tb4b4b4]Tb4b4b4, T79c0ff0Tb4b4b4)Tb4b4b4:

Tff7b72selfTff7b72.Td2a8ff_futuresTff7b72.Td2a8ffremoveTb4b4b4(Te6edf3futureTb4b4b4)

Tff7b72tryTb4b4b4:

Te6edf3result Tff7b72= Te6edf3futureTff7b72.Td2a8ffresultTb4b4b4(Tb4b4b4)

Tff7b72except Tf85149ExceptionTb4b4b4:

Te6edf3tracebackTff7b72.Td2a8ffprint_excTb4b4b4(Tb4b4b4)

T8b949e# make an error tile?

Tff7b72continue

Tff7b72if Te6edf3result Tff7b72is Tff7b72NoneTb4b4b4:

Tff7b72continue

Te6edf3callbackTb4b4b4, Te6edf3args Tff7b72= Te6edf3result

Te6edf3callbackTb4b4b4(Tff7b72*Te6edf3argsTb4b4b4)

T8b949e# capped executor in time, in order to prevent too much

T8b949e# slowiness.

T8b949e# seems to works quite great with big zoom-in/out

Tff7b72if Te6edf3timeTb4b4b4(Tb4b4b4) Tff7b72- Te6edf3start Tff7b72> Tff7b72selfTff7b72.Td2a8ffcap_timeTb4b4b4:

Tff7b72break

Tff7b72except Tf85149TimeoutErrorTb4b4b4:

Tff7b72pass